import os
import requests
import re
import csv
import traceback
import json
import datetime
from fake_useragent import UserAgent
from dbpool import get_connection_exception

DATETIME_FORMAT = '%Y%m%d%H%M%s'

ua = UserAgent()
CHARSET = 'gb2312'

class HtmlDownloader(object):

    def download(self, url):
        if url is None:
            return None
        # 生成随机User-Agent
        headers = {"User-Agent": ua.random}
        response = requests.get(url, headers=headers)
        if response.status_code != 200:
            return None
        response_text = response.text
        page_encode_re = re.search(r'charset=(.*)"', response_text, re.M | re.I)
        page_encode = page_encode_re.group(1)
        if page_encode.lower() == 'utf-8' or page_encode.lower() == 'utf8':
            response_text = response_text.encode("latin1").decode('utf-8')
            return response_text
        elif str(page_encode.lower()).find('gb') != -1:
            try:
                response_text = response_text.encode("latin1").decode('GB18030')
            except Exception as e:
                print(e)
                response_text = response_text.encode("latin1").decode('GBK')

            return response_text
        else:
            raise Exception('charset error')


def _get_class_soup(soup, class_name, num_n=1):
    """
    获取soup里下一个class的Tag值
    """
    for index, bs in enumerate(soup.find_all(class_=class_name)):
        if index+1 < num_n:
            continue
        else:
            return bs


def _get_next_soup(soup, tag, num_n=1):
    """
    获取soup里下一个tag的Tag值
    """
    n = 1
    for i in soup.next_siblings:
        if hasattr(i, 'name') and i.name == tag:
            if n == num_n:
                return i
            else:
                n += 1
    return None


def _get_inner_soup(soup, tag, num_n=1):
    """
    获取soup里面第num_n个Tag值
    """
    sb_list = list()
    if isinstance(soup, list):
        for s in soup:
            for sb in s.find_all(tag):
                if num_n == 0:
                    sb_list.append(sb)
                else:
                    raise Exception("num_n值必须为0")
    else:
        for index, sb in enumerate(soup.find_all(tag)):
            if num_n == 0:
                sb_list.append(sb)
            else:
                if index+1 == num_n:
                    return sb
    return sb_list


def get_soup(soup, class_tag, level=1):
    _type = class_tag[level - 1][0]
    _tag = class_tag[level - 1][1]
    _note = class_tag[level - 1][2]
    if _type == 'class':
        bs = _get_class_soup(soup, _tag, int(_note or 1))
    elif _type == 'next':
        bs = _get_next_soup(soup, _tag, int(_note))
    elif _type == 'inner':
        bs = _get_inner_soup(soup, _tag, int(_note))
    if level == len(class_tag):
        return bs
    else:
        return get_soup(bs, class_tag, level + 1)


def select_use_db_conf(db_conf):
    with get_connection_exception(db_conf['database']) as db:
        table = db_conf.get('table')
        fields = replace_str(db_conf.get('fields') or '', db_conf.get('replace'))
        where = replace_str(db_conf.get('where') or '', db_conf.get('replace'))
        other = replace_str(db_conf.get('other') or '', db_conf.get('replace'))
        datas = db.select(table=table,
                          fields=fields,
                          where=where,
                          other=other)
    if datas:
        return datas
    return None


def replace_str(s, replace_dict):
    """
    替换
    """
    if replace_dict and isinstance(replace_dict, dict):
        if isinstance(s, set):
            s = list(s)
        if isinstance(s, str):
            for k, v in replace_dict.items():
                s = re.sub(k, v, s)
        elif isinstance(s, list):
            for i in range(len(s)):
                s[i] = replace_str(s[i], replace_dict)
        elif isinstance(s, dict):
            for sk, sv in s.items():
                s[sk] = replace_str(sv, replace_dict)
    return s


def insert_batch(db, table, datas, num=2000):
    """
    批量插入
    """
    try:
        cnt = 0
        while len(datas) > cnt:
            ceil_list = datas[cnt:cnt + num]
            db.insert_list(table, ceil_list)
            cnt += num
    except Exception as e:
        print(traceback.format_exc())
        print(e)

def insert_batch_onebyone(db, table, datas):
    """
    批量插入
    """
    try:
        for data in datas:
            db.insert(table, data)
    except Exception as e:
        print(traceback.format_exc())
        print(e)


def save_file(datas, file_path):
    if file_path.endswith('txt') or file_path.endswith('TXT'):
        with open(file_path, 'a+') as txtfile:
            for row in datas:
                txtfile.write(str(row) + '\r\n')
    elif file_path.endswith('csv') or file_path.endswith('CSV'):
        if os.path.isfile(file_path):
            with open(file_path, 'a+') as csvfile:
                writer = csv.writer(csvfile)
                for row in datas:
                    writer.writerow(str(row))
